from selenium import webdriver
import time
import sys
from selenium.webdriver.chrome.options import Options

# 文件路径
fpath = 'C:\\data\\CNVD_demo2\\webBugs.json'

# 循环获取漏洞列表
from lxml import etree
def parse_list(base_url,driver):
    for i in range(160,175):
        start_url = 'https://www.cnvd.org.cn/flaw/typeResult?typeId=29&max=20&offset={}'.format(i*20)
        fp = open(fpath, 'a')
        driver.get(start_url)
        print('爬取列表页...', driver.current_url)
        time.sleep(2)
        html = etree.HTML(driver.page_source)
        detail_urls = html.xpath("//a[contains(@href,'CNVD')]/@href")
        for url in detail_urls:
            full_url = base_url + url
            parse_detail(full_url, base_url, fp, driver)
            time.sleep(3)
        fp.close()

# 爬取漏洞详情
def parse_detail(url,base_url,fp, driver):
    try:
        driver.get(url)
        html = driver.page_source
        print("爬取详情页...", driver.current_url)
        time.sleep(3)
        response = etree.HTML(html)

        #漏洞名称
        title = response.xpath("//div[@class='blkContainerSblk']/h1/text()")[0].strip()

        # 公开日期 str
        date = response.xpath("//table[@class='gg_detail']/tbody//tr[2]/td[2]/text()")[0].strip()

        # CNVD_ID str
        CN_ID = response.xpath("//table[@class='gg_detail']//tr[1]//td[2]/text()")[0].strip()

        # 产品 list
        projects_demo = response.xpath("//table[@class='gg_detail']/tbody//tr[4]/td[2]/text()")
        projects = []
        for demo in projects_demo:
            demo = demo.strip()
            if demo != '':
                projects.append(demo)

        # 描述 str
        contents = response.xpath(
            "//table[@class='gg_detail']//td[contains(text(), '漏洞描述')]/following-sibling::td[1]//text()")
        contents = ''.join([j.strip() for j in contents])

        # 等级 str
        level = response.xpath("//table[@class='gg_detail']/tbody//tr[3]/td[2]/text()[2]")
        if level:
            level = level[0].replace('(','').strip()
        else:
            level = '未评级'

        # 参考链接 url
        ref_url = response.xpath(
            "//table[@class='gg_detail']//td[contains(text(), '参考链接')]/following-sibling::td[1]/a/@href")

        # 解决方案 str
        solve_contents = response.xpath(
            "//table[@class='gg_detail']//td[contains(text(), '漏洞解决方案')]/following-sibling::td[1]//text()")
        solve_contents = ''.join([j.strip() for j in solve_contents])

        # 厂商补丁 str
        buding = response.xpath(
            "//table[@class='gg_detail']//td[contains(text(), '厂商补丁')]/following-sibling::td[1]/a/@href")
        if buding:
            if isinstance(buding[0], str):
                buding = base_url + buding[0]
        dic = {}
        dic['漏洞名称'] = title
        dic['CNVD-ID'] = CN_ID
        dic['公开日期'] = date
        dic['危害级别'] = level
        dic['影响产品'] = projects
        dic['漏洞描述'] = contents
        dic['参考链接'] = ref_url
        dic['漏洞解决方案'] = solve_contents
        dic['厂商补丁'] = buding
        save_item(dic,fp)
    except:
        print(">>>出现错误...",driver.current_url)
        print(driver.page_source)
        sys.exit(1)


# 保存单个漏洞信息
import json
def save_item(dic, fp):
    str_item = json.dumps(dic, ensure_ascii=False)
    fp.write(str_item+"\n")

# main函数
def main(driver):
    base_url = 'https://www.cnvd.org.cn'
    parse_list(base_url,driver)

if __name__ == '__main__':
    # 配置chrome信息
    chrome_option = Options()
    chrome_option.add_argument("--headless")
    driver = webdriver.Chrome(options=chrome_option)
    main(driver)
    driver.close()